package com.dm.webCrawler.project.buss.service;

import cn.hutool.core.date.DateUtil;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.net.URLDecoder;
import cn.hutool.core.util.ZipUtil;
import cn.hutool.http.HttpRequest;
import com.dm.webCrawler.common.utils.IdUtils;
import com.dm.webCrawler.common.utils.file.XmlDetailUtil;
import com.dm.webCrawler.common.utils.http.HttpHelper;
import com.dm.webCrawler.project.mapper.CrawlerIndexMapper;
import com.dm.webCrawler.project.mode.ZipDetialModel;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;

import javax.xml.bind.JAXBException;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.Date;
import java.util.Map;

@Component
public class SinaCrawler {
    private final String WEIBO_HOST="https://s.weibo.com";
    private final String RETIME="/realtime?q=${ht}&rd=realtime&tw=realtime&Refer=hot_realtime";
    private final String HOT="/hot?q=${ht}&xsort=hot&suball=1&tw=hotweibo&Refer=realtime_hot";
    @Value("${lb.out-zip-path:outZipPath}")
    private String outZipPath;
    @Value("${lb.work-back:news}")
    private String workBack;
    private String typeName="sinaweibo";
    public static enum searchType {
        /**
         * 热搜
         */
        HOT("?cate=realtimehot"),
        /**
         * 要闻
         */
        SOCIALEVENT("?cate=socialevent");
        private String type;
        private searchType(String type){
            this.type=type;
        }
        public String getType(){
            return this.type;
        }
    }

    @Autowired
    private CrawlerIndexMapper crawlerIndexMapper;


    public void start(searchType type){

        String body = HttpRequest.get("https://s.weibo.com/top/summary" + type.getType()).headerMap(headerMap(), true).execute().body();
        Document doc = Jsoup.parse(body);
        Elements eles = doc.select("#pl_top_realtimehot  tbody tr>td:nth-child(2)>a");
        if(eles!=null && eles.size()>0){
            for (Element ele : eles) {
                String url=WEIBO_HOST+ele.attr("href");
                try {
                    if("•".equals(ele.parent().previousElementSibling().text())){
                        continue;
                    }

                    String ht=ele.text();//话题
                    String basDir=workBack+"/"+typeName+"/"+ht;
                    File basDirFile = new File(basDir);
                    basDirFile.mkdirs();
                    String listZhbody = HttpRequest.get(url).headerMap(headerMap(), true).execute().body(); //综合
                    handlerHostList(listZhbody,ht,basDir);

//                    String listSsbody = HttpRequest.get(WEIBO_HOST+RETIME.replace("${ht}","%23"+ht+"%23")).headerMap(headerMap(), true).execute().body(); //实时
//                    handlerHostList(listSsbody,ht,basDir);

                    String listRmbody = HttpRequest.get(WEIBO_HOST+HOT.replace("${ht}","%23"+ht+"%23")).headerMap(headerMap(), true).execute().body(); //热门
                    handlerHostList(listRmbody,ht,basDir);
                    FileUtil.del(basDirFile.getAbsolutePath());
                }catch (Exception e){
                    e.printStackTrace();
                }

            }
        }
    }

    /**
     * 处理热门列表数据
     * @param listbody 页面
     * @param ht 话题
     * @param baseDir 根目录
     */
    private void handlerHostList(String listbody,String ht,String baseDir){

        Document doc = Jsoup.parse(listbody);
        Elements eles = doc.select(".card-wrap[action-type=feed_list_item]");
        if(eles!=null && eles.size()>0){
            for (Element ele : eles) {
                ZipDetialModel model=new ZipDetialModel();//输出
                String mid=ele.attr("mid");
                if(crawlerIndexMapper.hasIndex(mid,typeName)>=1){
                    return;
                }

                model.setWybs(mid); //设置唯一标识
                model.setLy("新浪微博");
                model.setCjfl(ht);
                model.setYz("中文");
                /**
                 * 取作者
                 */
                try {
                    Element userInfo = ele.select("a.name").first();
                    String nick_name=userInfo.text();
                    model.setBjr(nick_name);
                    model.setFbmt(nick_name);
                    model.setBt(nick_name+":"+ht);
                    model.setLydz("https:"+userInfo.attr("href"));
                }catch (Exception e){
                    e.printStackTrace();
                    System.out.println("获取用户信息失败");
                }
                /**
                 * 两个时间
                 */
                //发布时间还没找到采集办法

                //获取时间
                model.setHqsj(DateUtil.format(new Date(),"yyyy-MM-dd HH:mm:ss"));

                /**
                 * 四个数量
                 */
                try{
                    Elements numberli = ele.select("div.card>div.card-act>ul>li");
                    if(numberli!=null && !numberli.isEmpty() && numberli.size()==4) {
                        //转发数
                        String zfsStr=numberli.get(1).child(0).text().replace("转发","").trim();
                        Integer zfs=Integer.valueOf(null==zfsStr || "".equals(zfsStr)?"0":zfsStr);
                        model.setZfs(zfs);
                        //评论数
                        String plsStr=numberli.get(2).child(0).text().replace("评论","").trim();
                        Integer pls=Integer.valueOf(null==plsStr || "".equals(plsStr)?"0":plsStr);
                        model.setPls(pls);
                        //点赞数
                        String dzsStr=numberli.get(3).select("em").first().text();
                        Integer dzs=Integer.valueOf(null==dzsStr || "".equals(dzsStr)?"0":dzsStr);
                        model.setDzs(dzs);
                    }
                }catch (Exception e){
                    e.printStackTrace();
                    System.out.println("取数量失败");
                }
                String msgBasDir=baseDir+"/"+mid;
                new File(msgBasDir).mkdirs();
                Elements fulltxt = ele.select(".content>.txt[node-type=feed_list_content_full]");
                if(fulltxt==null || fulltxt.isEmpty()){
                    fulltxt=ele.select(".content>.txt[node-type=feed_list_content]");
                }
                //获取正文
                String content=fulltxt.text();
                String contentFileName="content.txt";
                File f=new File(msgBasDir+"/"+contentFileName);
                try {
                    f.createNewFile();
                    FileUtil.writeString(content,f,"UTF-8");
                    model.getWJ().add(contentFileName);
                } catch (IOException e) {
                    e.printStackTrace();
                    System.out.println("创建正文失败！");
                }
                //获取视频资源
                Elements video = ele.select(".content>.media.media-video-a[node-type=feed_list_media_prev]");
                if(video!=null && !video.isEmpty()){
                    String actionData = video.first().select("a.WB_video_h5[node-type=fl_h5_video]").first().attr("action-data");
                    String videoUrl=searchActionData(actionData,"video_src");
                    String videoFilePath=IdUtils.fastSimpleUUID()+".mp4";
                    File videoFile=new File(msgBasDir+"/"+ videoFilePath);
                    try {
                        videoFile.createNewFile();
                    } catch (IOException e) {
                        e.printStackTrace();
                        System.out.println("创建mp4文件失败");
                    }
                    try{
                        String decodeUrl=URLDecoder.decode(videoUrl, Charset.forName("UTF-8"));
                        HttpRequest.get(decodeUrl.startsWith("http")?decodeUrl:"https:"+decodeUrl).execute().writeBody(videoFile);
                        model.getFJ().add(videoFilePath);
                    }catch (Exception e){
                        e.printStackTrace();
                        System.out.println("下载视频文件失败:"+model);
                    }

                }
                //获取图片资源
                Elements imgs = ele.select(".content div.media.media-piclist[node-type=fl_pic_list]>ul>li>a");
                if(imgs!=null && !imgs.isEmpty()){
                    for (Element img : imgs) {
                        String imgFilePath=IdUtils.fastSimpleUUID()+".jpg";
                        File imgFile=new File(msgBasDir+"/"+ imgFilePath);
                        try {
                            imgFile.createNewFile();
                        } catch (IOException e) {
                            e.printStackTrace();
                            System.out.println("创建图片文件失败");
                        }
                        try {
                            HttpRequest.get("https:"+img.attr("src")).execute().writeBody(imgFile);
                            model.getFJ().add(imgFilePath);
                        }catch (Exception e){
                            e.printStackTrace();
                            System.out.println("下载图片失败");
                        }

                    }

                }
                //输出xml文件
                try {
                    XmlDetailUtil.createXML(model,msgBasDir+"/detail.xml");
                    //输出压缩文件
                    String outPath=outZipPath+"/"+typeName+"_"+ht+"_"+mid+".zip";
                    String basDirAbs=new File(msgBasDir).getAbsolutePath();
                    ZipUtil.zip(basDirAbs,new File(outPath).getAbsolutePath());
                    FileUtil.del(basDirAbs);
                    crawlerIndexMapper.appendIndex(mid,typeName);
                } catch (JAXBException e) {
                    e.printStackTrace();
                    System.out.println("创建压缩文件失败");
                }
            }
        }
    }



    private String searchActionData(String actionData,String key){
        Map<String, String> paramMaps = HttpHelper.getParamMaps(actionData);
        return paramMaps.get(key);
    }



    private Map<String,String> headerMap(){
        String headerString="Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9\n" +
                "Accept-Encoding: gzip, deflate, br\n" +
                "Accept-Language: zh-CN,zh;q=0.9\n" +
                "Cache-Control: max-age=0\n" +
                "Connection: keep-alive\n" +
//                "Cookie: login_sid_t=050d09edc1e4daa83b4c7d470b72a579; cross_origin_proto=SSL; _s_tentry=passport.weibo.com; Apache=4254416217140.928.1630237198874; SINAGLOBAL=4254416217140.928.1630237198874; ULV=1630237198877:1:1:1:4254416217140.928.1630237198874:; SUB=_2AkMWd_0vf8NxqwJRmP4Ry2zraYhwwgjEieKgKwz0JRMxHRl-yT9jqhZatRB6PffTwIELvovMc7XaZdw15JrJrr6Fbfjl; SUBP=0033WrSXqPxfM72-Ws9jqgMF55529P9D9WFU.Rqp2.vkomI32fuIw.dC; UOR=,,www.baidu.com\n" +
                "Host: s.weibo.com\n" +
                "Referer: https://s.weibo.com/top/summary?cate=socialevent\n" +
                "sec-ch-ua: \"Google Chrome\";v=\"89\", \"Chromium\";v=\"89\", \";Not A Brand\";v=\"99\"\n" +
                "sec-ch-ua-mobile: ?0\n" +
                "Sec-Fetch-Dest: document\n" +
                "Sec-Fetch-Mode: navigate\n" +
                "Sec-Fetch-Site: same-origin\n" +
                "Sec-Fetch-User: ?1\n" +
                "Upgrade-Insecure-Requests: 1\n" +
                "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36";
        return HttpHelper.convertHeaderMap(headerString);
    }


    public static void main(String[] args) {
        new SinaCrawler().start(searchType.HOT);
    }

}
